# ============================== SETUP ===============================
rm(list = ls())
options(scipen = 999)
setwd("~/Dropbox/Wayne-Ying/White_Nationalist_Recruitment/replication/codes")

intercodergab <- read.csv("../datasets/input/intercodergab.csv", stringsAsFactors = FALSE)
intercodergab <- intercodergab[order(intercodergab$id),]
intercodergab1 <- intercodergab[seq(1, nrow(intercodergab), by = 2),]
intercodergab2 <- intercodergab[seq(2, nrow(intercodergab), by = 2),]

intercodertwitter <- read.csv("../datasets/input/intercodertwitter.csv", stringsAsFactors = FALSE)
intercodertwitter <- intercodertwitter[order(intercodertwitter$id),]
intercodertwitter1 <- intercodertwitter[seq(1, nrow(intercodertwitter), by = 2),]
intercodertwitter2 <- intercodertwitter[seq(2, nrow(intercodertwitter), by = 2),]

IRrace <- (sum(intercodergab1$Ethnicity.Race==intercodergab2$Ethnicity.Race)+sum(intercodertwitter1$Ethnicity.Race==intercodertwitter2$Ethnicity.Race))/(540+3000)
IRnat <- (sum(intercodergab1$Nationalism==intercodergab2$Nationalism)+sum(intercodertwitter1$Nationalism==intercodertwitter2$Nationalism))/(540+3000)
IRgender <- sum(intercodergab1$Gender==intercodergab2$Gender)/540
IRparty <- (sum(intercodergab1$Partisan==intercodergab2$Partisan)+sum(intercodertwitter1$Partisan==intercodertwitter2$Partisan))/(540+3000)
IRrelig <- (sum(intercodergab1$Religion==intercodergab2$Religion)+sum(intercodertwitter1$Religion==intercodertwitter2$Religion))/(540+3000)

stargazer::stargazer(t(as.matrix(c(IRrace, IRnat, IRgender, IRparty, IRrelig))), 
                     column.labels = c("Race", "Nationalism", "Gender", "Partisan", "Religion"),
                     summary = FALSE)


# ============================== Compare to AI coding ===============================

library(dplyr)
library(tidyr)
library(knitr)

data_df <- read.csv("../datasets/input/compareLLMs.csv",
                    header = TRUE, stringsAsFactors = FALSE, check.names = FALSE)

desired_categories <- c("Relevant","Racist","Nationalist","Gender","Partisan","Religious")
desired_methods <- c("AD_Logit","Keyword","LlaMa","Gpt 3.5 turbo-0125","Gpt 4 turbo",
                     "Gpt 4o","Gemini 1.5-flash","Mistral Small","Mistral Large")

get_accuracy <- function(pred, actual) {
  keep <- !is.na(pred) & !is.na(actual)
  pred <- pred[keep]; actual <- actual[keep]
  TP <- sum(pred == 1 & actual == 1)
  FP <- sum(pred == 1 & actual == 0)
  TN <- sum(pred == 0 & actual == 0)
  FN <- sum(pred == 0 & actual == 1)
  (TP + TN) / (TP + TN + FP + FN)
}

results_list <- list()

for (mtd in desired_methods) {
  for (catg in desired_categories) {
    method_col <- paste0(mtd, "__", catg)
    human_col  <- paste0("Human__", catg)
    
    if (!all(c(method_col, human_col) %in% colnames(data_df))) {
      out <- data.frame(accuracy = NA_real_, method = mtd, category = catg)
      results_list[[paste(mtd, catg, sep = "_")]] <- out
      next
    }
    
    acc <- get_accuracy(data_df[[method_col]], data_df[[human_col]])
    out <- data.frame(accuracy = acc, method = mtd, category = catg)
    results_list[[paste(mtd, catg, sep = "_")]] <- out
  }
}

accuracy_df <- dplyr::bind_rows(results_list) |>
  dplyr::mutate(
    method   = factor(method,   levels = desired_methods),
    category = factor(category, levels = desired_categories)
  ) |>
  dplyr::arrange(method, category)

accuracy_table <- accuracy_df |>
  dplyr::select(method, category, accuracy) |>
  tidyr::pivot_wider(names_from = category, values_from = accuracy) |>
  dplyr::arrange(method)

print(
  knitr::kable(
    accuracy_table,
    format   = "latex",
    booktabs = TRUE,
    caption  = "Accuracy by method and category (vs. Human)."
  )
)